In [1]:
%load_ext autoreload
%autoreload 2
In [2]:
import sys
sys.path.append('.')
from datetime import datetime
import numpy as np
import matplotlib.pyplot as plt

import pandas as pd

import incense
from incense import ExperimentLoader
from cycler import cycler

Finding experiments

To use incense we first have to instantiate an experiment loader that will enable us to query the database for specific runs.

In [3]:
loader = ExperimentLoader(
    mongo_uri="mongodb://localhost:27017", 
    db_name='sacred'
)
In [4]:
query = {"$and": [
            {"config.autoencoder_type": "Over_dim"},
           
        ]}
experiments=loader.find(query)
In [5]:
experiments
Out[5]:
QuerySet([Experiment(id=1, name=autoencoder_test), Experiment(id=2, name=autoencoder_test), Experiment(id=3, name=autoencoder_test), Experiment(id=4, name=autoencoder_test), Experiment(id=5, name=autoencoder_test), Experiment(id=6, name=autoencoder_test), Experiment(id=7, name=autoencoder_test), Experiment(id=8, name=autoencoder_test), Experiment(id=66, name=autoencoder_test), Experiment(id=67, name=autoencoder_test), Experiment(id=68, name=autoencoder_test), Experiment(id=69, name=autoencoder_test)])
In [6]:
experiments[0].config
Out[6]:
pmap({'batch_size': 256, 'epochs': 250, 'autoencoder_type': 'Over_dim', 'iteration': False, 'seed': 677719408, 'targets_type': '10_Targets'})
In [7]:
experiments.project(on=["config.targets_type","config.targets_type", "config.iteration", "config.autoencoder_type", "config.batch_size"])
Out[7]:
targets_type iteration autoencoder_type batch_size
exp_id
1 10_Targets False Over_dim 256
2 10_Targets False Over_dim 128
3 10_Targets False Over_dim 64
4 10_Targets False Over_dim 32
5 Mnist False Over_dim 256
6 Mnist False Over_dim 128
7 Mnist False Over_dim 64
8 Mnist False Over_dim 32
66 Noisy False Over_dim 256
67 Noisy False Over_dim 128
68 Noisy False Over_dim 64
69 Noisy False Over_dim 32
In [8]:
#experiments=experiments[0:6]
experiments
Out[8]:
QuerySet([Experiment(id=1, name=autoencoder_test), Experiment(id=2, name=autoencoder_test), Experiment(id=3, name=autoencoder_test), Experiment(id=4, name=autoencoder_test), Experiment(id=5, name=autoencoder_test), Experiment(id=6, name=autoencoder_test), Experiment(id=7, name=autoencoder_test), Experiment(id=8, name=autoencoder_test), Experiment(id=66, name=autoencoder_test), Experiment(id=67, name=autoencoder_test), Experiment(id=68, name=autoencoder_test), Experiment(id=69, name=autoencoder_test)])
In [9]:
def print_imm(imgs,name):
    n = len(imgs[0]) # how many digits we will display
    if name:
        plt.figure(figsize=(2,0.5))
        plt.text(0.1, 0.1, name, fontsize=12) 
        plt.show()
    plt.figure(figsize=(2*len(imgs[0]), 2 * len(imgs) + 2))
    for i in range(n):
        for j in range(len(imgs)):
            # display original
            ax = plt.subplot(len(imgs), n, i + 1 + j * n)
            
            plt.imshow(imgs[j][i].reshape(28, 28))
            
            plt.gray()
            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(False)

    plt.show()
In [10]:
def compare(data):
    evaluations_in_classifier=[]
    evaluations_feature_classifier=[]
    evaluations_out_classifier=[]
    autoencoder=[]
    
    for exp in experiments:
        #print(exp.id,exp.config)
        pickle_artifact = exp.artifacts[data].as_type(incense.artifact.PickleArtifact)
        predictions=pd.read_pickle(pickle_artifact.file,compression='gzip')
        
        evaluations_in_classifier.append(predictions['evaluations_in_classifier'])
        evaluations_feature_classifier.append(predictions['evaluations_feature_classifier'])
        evaluations_out_classifier.append(predictions['evaluations_out_classifier'])
        autoencoder.append(predictions['evaluations_autoencoder'])
        

    print(data)
    names=[]
    for exp in experiments:
        names.append(exp.id)
    
    plt.rc('axes', prop_cycle=(cycler('color', ['r', 'g', 'b', 'y','r', 'g', 'b', 'y','r', 'g', 'b', 'y','r', 'g', 'b', 'y']) +
                           cycler('linestyle', ['-','-','-','-', '--','--','--','--', ':',':',':',':' ,'-.','-.','-.','-.'])))
    
    plt.figure(figsize=(30,15))    

    plt.subplot(2, 3,1)
    plt.title("Accuracy over iterations evaluations_in_classifier") 
    i=0
    for exp in evaluations_in_classifier:
        plt.plot([item[1] for item in exp], label=names[i])
        i=i+1
    plt.legend()
    
    plt.subplot(2, 3,2)
    plt.title("Accuracy over iterations evaluations_feature_classifier") 
    i=0
    for exp in evaluations_feature_classifier:
        plt.plot([item[1] for item in exp], label=names[i])
        i=i+1
    plt.legend()
    
    
    plt.subplot(2, 3,3)
    plt.title("Accuracy over iterations evaluations_out_classifier") 
    i=0
    for exp in evaluations_out_classifier:
        plt.plot([item[1] for item in exp], label=names[i])
        i=i+1
    plt.legend()  

    plt.subplot(2, 3,4)
    plt.title("Loss over iterations autoencoder") 
    i=0
    for exp in autoencoder:
        plt.plot([item[0] for item in exp], label=names[i])
        i=i+1
    plt.legend()
    
  
    
    
    plt.subplot(2, 3,6)
    plt.title("mae iterations autoencoder") 
    i=0
    for exp in autoencoder:
        plt.plot([item[2] for item in exp], label=names[i])
        i=i+1
    plt.legend()
    
    plt.show()
    
    
In [11]:
name_list=['predictions_df_0','predictions_df_10','predictions_df_20','predictions_df_30','predictions_df_40','predictions_df_50','predictions_df_60','predictions_df_70','predictions_df_80','predictions_df_90','predictions_df_100']
#name_list=['predictions_df_0','predictions_df_10','predictions_df_20']
In [12]:
#df = pd.concat([pd.read_pickle(exp.artifacts['predictions_df_0'].as_type(incense.artifact.PickleArtifact).file,compression='gzip')['predictions'] for exp in experiments],axis=1,keys=[str(exp.id) for exp in experiments])
#df
In [13]:
for index in name_list:
    compare(index)
predictions_df_0
predictions_df_10
predictions_df_20
predictions_df_30
predictions_df_40
predictions_df_50
predictions_df_60
predictions_df_70
predictions_df_80
predictions_df_90
predictions_df_100
In [17]:
for data in name_list:
    df=[]
    plt.figure(figsize=(2,0.5))
    plt.text(0.1, 0.1, data, fontsize=12) 
    plt.show()
    num=['7','2','1','0']
    df = pd.concat([pd.read_pickle(exp.artifacts[data].as_type(incense.artifact.PickleArtifact).file,compression='gzip')['predictions'] for exp in experiments],axis=1,keys=[str(exp.id) for exp in experiments])
    for i in range(2):
        print_imm(np.array(df.applymap(lambda x: x[i])),num[i])
    df=[]